const puppeteer = require('puppeteer');
const MongoClient = require('mongodb').MongoClient;
const ObjectId = require('mongodb').ObjectId;

const url = "mongodb://taosha_user:apyl8XWP6LSW4JHs@101.35.6.34:27722/taosha";


async function main() {
  const browser = await puppeteer.launch({
    headless: false,
    args: ['--no-sandbox']
  });

  let page = await browser.newPage();
  await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});


  const client = await MongoClient.connect(url, {useNewUrlParser: true, useUnifiedTopology: true});
  const db = client.db('taosha');

  let topics = await db.collection('topics')
    .find({author_name: '知乎日报', pic: {$exists: false}})
    .sort({_id: -1}).skip(0).limit(20000)
    .toArray();

  let len = topics.length;
  console.log('len:', len);

  for (let i = 0; i < topics.length; i++) {
    let topic = topics[i];


    try {
      await page.goto(topic.original_url, {waitUntil: 'domcontentloaded'});
      await page.waitForSelector('.DailyHeader-image img', {timeout: 1000});
      let title = await page.title();
      if (title == '404 - 知乎') {
        let result = await db.collection('topics').deleteOne({_id: topic._id});
        console.log('删除:', i, len, topic.content);
      } else {
        topic.pic = await page.$eval('.DailyHeader-image img', el => el.src);
        topic.intro = await page.$eval('.content', el => el.innerText);
        if (topic.intro) {
          topic.intro = topic.intro.substr(0, 100);
        }
        if (!topic.pic) {
          console.error(topic.original_url, ' 图片没有获取到');
        }
        if (!topic.intro) {
          console.error(topic.original_url, ' 简介没有获取到');
        }


        let result = await db.collection('topics').updateOne({_id: topic._id}, {
          $set: {
            intro: topic.intro,
            pic: topic.pic,
          }
        });
        console.log(topic.title, result.result, i, len);
      }


    } catch (e) {
      console.error(topic.original_url, e.message);
      let result = await db.collection('topics').deleteOne({_id: topic._id});
      console.log('222删除:', i, len, topic.content);
    } finally {

    }
  }


}

main();
